#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import time,datetime
import queue
from datetime import date
from core.spiderThread import spiderThread
from models import domain

urls_maxCount=200
urls_minCount=110
urls=queue.Queue(urls_maxCount)

sd=spiderThread(2,2)

def onComplete():
    if urls.qsize()<urls_minCount:
        result = domain.domain().find(query={"lastgettime":{"$lte":datetime.datetime.utcnow()+datetime.timedelta(minutes=-30)}},limit=urls_maxCount-urls.qsize())
        for item in result:
            url=item["name"]
            domain.domain().update({"name":url},{"lastgettime":datetime.datetime.utcnow()})
            urls.put(url)
    for i in range(0,sd.__poollist__.getCount()):
        if urls.qsize()>=1:
            sd.addTask(urls.get(),"geturl")
sd.addlisten("COMPLETE",onComplete)
def main(): 
    if urls.qsize()<urls_minCount:
        result = domain.domain().find(query={},limit=urls_maxCount-urls.qsize())
        for item in result:
            urls.put(item["name"])
    if urls.qsize()>0:
        for i in range(0,sd.__poollist__.getCount()):
            if urls.qsize()>=1:
                sd.addTask(urls.get(),"geturl")
    else:
        pass
        sd.addTask("http://news.baidu.com","geturl")
main()